# -*- coding: utf-8 -*-
# @Time: 2021/5/15 10:10
# @Author: moxinqian
# @Contact: tel 18382469064
# @File: house_lianjia.py
# @DESC: 链家二手房房源爬取
import scrapy
from house_spider.items import *

host = 'https://cd.fang.lianjia.com'

lp_type = '链家新房'


class House(scrapy.Spider):
    name = "house_lianjia"

    custom_settings = {
        # 日志的输出级别
        "LOG_LEVEL": 'WARNING',
        # 最大延迟
        "MAX_RANDOM_DELAY": 5,
        # 最小延迟
        "MIN_RANDOM_DELAY": 1,
        "ROBOTSTXT_OBEY": False,
        "DOWNLOADER_MIDDLEWARES": {
            "house_spider.middlewares.RandomDelayMiddleware": 300,
            'house_spider.middlewares.SeleniumRemoteDownloaderMiddleware': 301
        },
        "ITEM_PIPELINES": {
            'house_spider.pipelines.LianjiaHouseMainInfoPipeline': 302
        }
    }

    # 启动页面
    def start_requests(self):
        #爬取2页数据用作测试
        for i in range(100):
            url = f'{host}/loupan/pg{i+1}'
            print(f"current url {url}")
            yield scrapy.Request(url=url, callback=self.parse)


    # 解析参数
    def parse(self, response, **kwargs):
        _item = LianJia_HouseMainInfoItem()
        print(f"scrapy url {response._url}")
        items = response.css('ul.resblock-list-wrapper li')
        for item in items:
            # print(item.get())
            _item['name'] = item.css('div.resblock-name > a::text').get()
            _item['lp_links'] = host + str(item.css('div.resblock-name > a::attr(href)').get())
            _item['position'] = item.css('div.resblock-location span::text').getall()
            _item['lp_status'] = item.css('div.resblock-name span::text').getall()
            _item['price_pre_m'] = item.css('div.main-price > span.number::text').getall()
            _item['total_price'] = item.css('div.resblock-price > div.second::text').get()
            _item['area'] = item.css('div.resblock-area>span::text').get()
            _item['lp_huxing'] = item.css('a.resblock-room>span::text').getall()
            _item['lp_features'] = item.css('div.resblock-tag span::text').getall()
            _item['lp_adviser'] = item.css('span.ke-agent-sj-name::text').get()
            _item['lp_type'] = lp_type
            #打印到控制台便于调试
            print(_item)
            #发送到pipline
            yield _item
